<!DOCTYPE html>
<html lang="en-us">
<head><head>
    <meta name="google-site-verification" content="9vIieCe-Qpd78QOmBl63rGtIVbhY6sYyuxX3j8XWBA4" />
    <meta name="baidu-site-verification" content="LRrmH41lz7" />
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="google-site-verification" content="xBT4GhYoi5qRD5tr338pgPM5OWHHIDR6mNg1a3euekI" />
    <meta name="viewport" content="width=device-width, initial-scale=1">
    
    <meta name="description" content="ceph从入门到放弃---ceph安装">
    
    <meta name="keyword"  content=", , 马少勋的网络日志, 马少勋的博客, Mashaoxun Blog, 博客, 个人网站, 互联网, Web, 云原生, PaaS, Istio, Kubernetes, 微服务, Microservice">
    <link rel="shortcut icon" href="/blog/img/favicon.ico">

    <title>ceph安装-马少勋的博客 | MaShaoXun Blog</title>

    <link rel="canonical" href="/blog/2019/01/30/ceph-install/">

    <link rel="stylesheet" href="/blog/css/iDisqus.min.css"/>
	
    
    <link rel="stylesheet" href="/blog/css/bootstrap.min.css">

    
    <link rel="stylesheet" href="/blog/css/hux-blog.min.css">

    
    <link rel="stylesheet" href="/blog/css/syntax.css">

    
    <link rel="stylesheet" href="/blog/css/zanshang.css">

    
    <link href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" type="text/css">
    
    
    <script src="/blog/js/jquery.min.js"></script>
    
    
    <script src="/blog/js/bootstrap.min.js"></script>
    
    
    <script src="/blog/js/hux-blog.min.js"></script>
	
    
    
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/styles/github.min.css">
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/highlight.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/dockerfile.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/vim.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/yaml.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/go.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/bash.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/shell.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/nginx.min.js"></script>
    <script>hljs.initHighlightingOnLoad();</script>
</head>
</head>

<nav class="navbar navbar-default navbar-custom navbar-fixed-top">
    <div class="container-fluid">
        
        <div class="navbar-header page-scroll">
            <button type="button" class="navbar-toggle">
                <span class="sr-only">Toggle navigation</span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
            </button>
            <a class="navbar-brand" href="/">Mashaoxun Blog</a>
        </div>

        
        
        <div id="huxblog_navbar">
            <div class="navbar-collapse">
                <ul class="nav navbar-nav navbar-right">
                    <li>
                        <a href="/blog">Home</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/django">django</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/docker">docker</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/k8s">k8s</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/redis">redis</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/swarm">swarm</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/%E5%AD%98%E5%82%A8">存储</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/%E7%9B%91%E6%8E%A7">监控</a>
                    </li>
                    
                    
		    
                        <li><a href="/blog/top/about/">ABOUT</a></li>
                    

                    
		    <li>
                        <a href="/blog/search">SEARCH <img src="/blog/img/search.png" height="15" style="cursor: pointer;"></a>
		    </li>
                    
                </ul>
            </div>
        </div>
        
    </div>
    
</nav>
<script>
    
    
    
    var $body   = document.body;
    var $toggle = document.querySelector('.navbar-toggle');
    var $navbar = document.querySelector('#huxblog_navbar');
    var $collapse = document.querySelector('.navbar-collapse');

    $toggle.addEventListener('click', handleMagic)
    function handleMagic(e){
        if ($navbar.className.indexOf('in') > 0) {
        
            $navbar.className = " ";
            
            setTimeout(function(){
                
                if($navbar.className.indexOf('in') < 0) {
                    $collapse.style.height = "0px"
                }
            },400)
        }else{
        
            $collapse.style.height = "auto"
            $navbar.className += " in";
        }
    }
</script>




<style type="text/css">
    header.intro-header {
        background-image: url('/blog/img/youyou.jpg')
    }
</style>
<header class="intro-header">
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <div class="post-heading">
                    <div class="tags">
                        
                        <a class="tag" href="/blog/tags/%E5%AD%98%E5%82%A8" title="存储">
                            存储
                        </a>
                        
                        <a class="tag" href="/blog/tags/ceph" title="ceph">
                            ceph
                        </a>
                        
                    </div>
                    <h1>ceph安装</h1>
                    <h2 class="subheading"></h2>
                    <span class="meta">
			Posted by 
			
			         &#34;马少勋&#34;
			 
			on 
			Wednesday, January 30, 2019
                        
                    </span>
                </div>
            </div>
        </div>
    </div>
</header>




<article>
    <div class="container">
        <div class="row">

            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                post-container">

                
                <header>
                    <h2>TOC</h2>
                </header>
                <nav id="TableOfContents">
<ul>
<li>
<ul>
<li><a href="#主机列表及host配置">主机列表及host配置</a></li>
<li><a href="#前期准备">前期准备</a>
<ul>
<li><a href="#添加免密登录-管理节点操作">添加免密登录(管理节点操作)</a></li>
<li><a href="#在各个节点上配置需要的yum源">在各个节点上配置需要的yum源</a></li>
</ul></li>
<li><a href="#安装ceph">安装ceph</a>
<ul>
<li><a href="#管理节点安装ceph-deploy工具-管理节点执行">管理节点安装ceph-deploy工具(管理节点执行)</a></li>
</ul></li>
<li><a href="#部署ceph集群-管理节点执行">部署ceph集群(管理节点执行)</a>
<ul>
<li><a href="#添加工作目录">添加工作目录</a></li>
<li><a href="#添加初始节点-管理节点执行">添加初始节点(管理节点执行)</a></li>
<li><a href="#管理节点添加管理网段配置">管理节点添加管理网段配置</a></li>
<li><a href="#部署initial-monitor">部署initial monitor</a></li>
<li><a href="#创建ceph-keyring">创建ceph keyring</a></li>
<li><a href="#分发keyring">分发keyring</a></li>
<li><a href="#创建mgr">创建mgr</a></li>
<li><a href="#创建osd">创建osd</a></li>
</ul></li>
<li><a href="#集群扩容">集群扩容</a>
<ul>
<li><a href="#添加两个osd-可以单台主机多块osd-实验为一台主机一块盘">添加两个osd(可以单台主机多块osd，实验为一台主机一块盘)</a></li>
<li><a href="#添加-2-mon-2-mgr">添加 2 mon + 2 mgr</a></li>
<li><a href="#添加一个mds-用于支持cephfs存储元数据">添加一个mds(用于支持cephfs存储元数据)</a></li>
</ul></li>
<li><a href="#增加client">增加client</a>
<ul>
<li><a href="#增加rgw-管理节点操作">增加rgw（管理节点操作）</a></li>
<li><a href="#增加cephfs">增加cephfs</a>
<ul>
<li><a href="#创建ceph-filesystem">创建ceph filesystem</a></li>
<li><a href="#通过ceph-fuse挂载">通过ceph fuse挂载</a></li>
<li><a href="#通过ceph-kernel-driver挂载">通过ceph kernel driver挂载</a></li>
</ul></li>
<li><a href="#增加rbd">增加rbd</a></li>
</ul></li>
</ul></li>
</ul>
</nav>
                
                

<h2 id="主机列表及host配置">主机列表及host配置</h2>

<table>
<thead>
<tr>
<th>ip地址</th>
<th>host</th>
</tr>
</thead>

<tbody>
<tr>
<td>10.19.128.253</td>
<td>ceph-node1</td>
</tr>

<tr>
<td>10.19.87.204</td>
<td>ceph-node2</td>
</tr>

<tr>
<td>10.19.116.54</td>
<td>ceph-node3</td>
</tr>
</tbody>
</table>

<pre><code>(node1) hostnamectl set-hostname ceph-node1
(node2) hostnamectl set-hostname ceph-node2
(node3) hostnamectl set-hostname ceph-node3
</code></pre>

<h2 id="前期准备">前期准备</h2>

<pre><code>yum install  -y  ntp ntpdate ntp-doc
systemctl start ntpd.service
systemctl enable ntpd.service
systemctl stop firewalld
systemctl disable firewalld
</code></pre>

<h3 id="添加免密登录-管理节点操作">添加免密登录(管理节点操作)</h3>

<pre><code>ssh-keygen -t rsa
ssh-copy-id root@ceph-node1
ssh-copy-id root@ceph-node2
ssh-copy-id root@ceph-node3
</code></pre>

<h3 id="在各个节点上配置需要的yum源">在各个节点上配置需要的yum源</h3>

<p>下载阿里base源</p>

<pre><code>wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
</code></pre>

<p>下载阿里epel源</p>

<pre><code>wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
</code></pre>

<p>添加ceph源</p>

<pre><code>cat /etc/yum.repos.d/ceph.repo
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/
gpgcheck=0
priority =1
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/
gpgcheck=0
priority =1
[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS
gpgcheck=0
priority=1
</code></pre>

<h2 id="安装ceph">安装ceph</h2>

<h3 id="管理节点安装ceph-deploy工具-管理节点执行">管理节点安装ceph-deploy工具(管理节点执行)</h3>

<pre><code>yum -y install python-setuptools
yum -y install ceph-deploy
ceph-deploy install ceph-node1 ceph-node2 ceph-node3
</code></pre>

<h2 id="部署ceph集群-管理节点执行">部署ceph集群(管理节点执行)</h2>

<p>为了演示<br />
1，创建一个集群：1mon+1mgr作为initial monitor<br />
2，添加多块osd，增加集群容量<br />
3，再次添加2mon+2mgr<br />
4，创建mds，作为元数据存储</p>

<h3 id="添加工作目录">添加工作目录</h3>

<pre><code>cd /etc/ceph
</code></pre>

<h3 id="添加初始节点-管理节点执行">添加初始节点(管理节点执行)</h3>

<p>初始节点可以是多个，为了演示扩容，初始先创建一个节点</p>

<pre><code>ceph-deploy new ceph-node3
</code></pre>

<h3 id="管理节点添加管理网段配置">管理节点添加管理网段配置</h3>

<pre><code>echo &quot;public network =10.19.0.0/16&quot; &gt;&gt; /etc/ceph/ceph.conf
</code></pre>

<h3 id="部署initial-monitor">部署initial monitor</h3>

<pre><code>ceph-deploy mon create ceph-node3
</code></pre>

<h3 id="创建ceph-keyring">创建ceph keyring</h3>

<pre><code>ceph-deploy gatherkeys ceph-node3
</code></pre>

<h3 id="分发keyring">分发keyring</h3>

<pre><code>ceph-deploy admin ceph-node1 ceph-node2 ceph-node3
</code></pre>

<h3 id="创建mgr">创建mgr</h3>

<pre><code>ceph-deploy mgr create ceph-node3
[root@ceph-node1 ceph]# ceph -s
  cluster:
    id:     b816c88c-2b8d-4a7f-b1f4-9a7e5f8ce9b6
    health: HEALTH_OK

  services:
    mon: 1 daemons, quorum ceph-node3
    mgr: ceph-node3(active)   ---&gt; #新添加的mgr
    osd: 0 osds: 0 up, 0 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0B
    usage:   0B used, 0B / 0B avail
    pgs:
</code></pre>

<h3 id="创建osd">创建osd</h3>

<pre><code>ceph-deploy osd create ceph-node3 --data /dev/vdb
[root@ceph-node1 ceph]# ceph -s
  cluster:
    id:     b816c88c-2b8d-4a7f-b1f4-9a7e5f8ce9b6
    health: HEALTH_OK

  services:
    mon: 1 daemons, quorum ceph-node3
    mgr: ceph-node3(active)
    osd: 1 osds: 1 up, 1 in ---&gt; #新添加的osd

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0B
    usage:   1.00GiB used, 99.0GiB / 100GiB avail
    pgs:
</code></pre>

<h2 id="集群扩容">集群扩容</h2>

<h3 id="添加两个osd-可以单台主机多块osd-实验为一台主机一块盘">添加两个osd(可以单台主机多块osd，实验为一台主机一块盘)</h3>

<pre><code>ceph-deploy osd create ceph-node1 --data /dev/vdb
ceph-deploy osd create ceph-node2 --data /dev/vdb
[root@ceph-node1 ceph]# ceph -s
  cluster:
    id:     b816c88c-2b8d-4a7f-b1f4-9a7e5f8ce9b6
    health: HEALTH_OK

  services:
    mon: 1 daemons, quorum ceph-node3
    mgr: ceph-node3(active)
    osd: 3 osds: 3 up, 3 in ---&gt; #一共三块osd

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0B
    usage:   3.01GiB used, 697GiB / 700GiB avail
    pgs:
</code></pre>

<h3 id="添加-2-mon-2-mgr">添加 2 mon + 2 mgr</h3>

<pre><code>ceph-deploy mon add ceph-node1
ceph-deploy mgr create ceph-node1
ceph-deploy mon add ceph-node2
ceph-deploy mgr create ceph-node2
[root@ceph-node1 ceph]# ceph -s
  cluster:
    id:     b816c88c-2b8d-4a7f-b1f4-9a7e5f8ce9b6
    health: HEALTH_WARN
            clock skew detected on mon.ceph-node3, mon.ceph-node1

  services:
    mon: 3 daemons, quorum ceph-node2,ceph-node3,ceph-node1 ---&gt; #mon三个
    mgr: ceph-node3(active), standbys: ceph-node1, ceph-node2 ---&gt; #mgr三个
    osd: 3 osds: 3 up, 3 in ---&gt; #服务器上有多块盘，可以添加多个osd

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0B
    usage:   3.01GiB used, 697GiB / 700GiB avail
    pgs:
</code></pre>

<h3 id="添加一个mds-用于支持cephfs存储元数据">添加一个mds(用于支持cephfs存储元数据)</h3>

<pre><code>ceph-deploy mds create ceph-node1
</code></pre>

<p>这次通过ceph -s 看不到变化，直到创建ceph filesystem的时候，mds才进入active状态<br />
可以去ceph-node1上查看进程ps -ef | grep ceph-mds<br />
至此，ceph集群就部署完了。<br />
由于上一章讲到，本地形式的RBD,RGW,CephFS和接口形式的librados都属于client，用于不同方式来访问ceph集群的数据。<br />
下面，为集群增加client</p>

<h2 id="增加client">增加client</h2>

<h3 id="增加rgw-管理节点操作">增加rgw（管理节点操作）</h3>

<p>我们可以使用ceph集群之外的server来部署rgw，为了方便起见，我们复用集群内的ceph-node1和ceph-node2来部署rgw。<br />
由于这三台主机都装了ceph包，默认自带ceph-radosgw，所以直接执行增加rgw命令。<br />
成功之后可以去相应主机ps -ef| grep radosgw 查看进程。<br />
并且可以通过http来访问curl ceph-node1:7480</p>

<pre><code>ceph-deploy rgw create ceph-node1 ceph-node2
[root@ceph-node1 ceph]# ps -ef| grep radosgw
ceph       41809       1  0 17:27 ?        00:00:00 /usr/bin/radosgw -f --cluster ceph --name client.rgw.ceph-node1 --setuser ceph --setgroup ceph
[root@ceph-node1 ceph]# curl ceph-node2:7480
&lt;?xml version=&quot;1.0&quot; encoding=&quot;UTF-8&quot;?&gt;&lt;ListAllMyBucketsResult xmlns=&quot;http://s3.amazonaws.com/doc/2006-03-01/&quot;&gt;&lt;Owner&gt;&lt;ID&gt;anonymous&lt;/ID&gt;&lt;DisplayName&gt;&lt;/DisplayName&gt;&lt;/Owner&gt;&lt;Buckets&gt;&lt;/Buckets&gt;&lt;/ListAllMyBucketsResult
</code></pre>

<p>单独主机部署的话需要安装ceph-radosgw包并关闭防火墙，然后再增加rgw，本实验忽略这步。</p>

<pre><code>ceph-deploy install --rgw &lt;client-node&gt; [&lt;client-node&gt; ...]
</code></pre>

<h3 id="增加cephfs">增加cephfs</h3>

<p>ceph filesystem需要mds，上面已经部署。并且，有两种方式来挂载：ceph fuse和ceph kernel driver。<br />
在这一节，我们:<br />
1.创建一个ceph filesystem<br />
2.通过ceph fuse挂载<br />
3.通过ceph kernel driver挂载</p>

<h4 id="创建ceph-filesystem">创建ceph filesystem</h4>

<p>创建所需的pool</p>

<pre><code>ceph osd pool create cephfs_data 80
ceph osd pool create cephfs_metadata 40
</code></pre>

<p>创建filesystem</p>

<pre><code>ceph fs new mycephfs cephfs_metadata cephfs_data
</code></pre>

<p>此时可以看到mds处于active状态了</p>

<pre><code>[root@ceph-node1 ceph]# ceph -s
  cluster:
    id:     b816c88c-2b8d-4a7f-b1f4-9a7e5f8ce9b6
    health: HEALTH_WARN
            clock skew detected on mon.ceph-node1

  services:
    mon: 3 daemons, quorum ceph-node2,ceph-node3,ceph-node1
    mgr: ceph-node3(active), standbys: ceph-node1, ceph-node2
    mds: mycephfs-1/1/1 up  {0=ceph-node1=up:active} ---&gt; #创建filesystem之后mds显示active了
    osd: 3 osds: 3 up, 3 in
    rgw: 2 daemons active

  data:
    pools:   6 pools, 152 pgs
    objects: 206 objects, 3.25KiB
    usage:   3.02GiB used, 697GiB / 700GiB avail
    pgs:     151 active+clean
             1   active+undersized

  io:
    client:   716B/s wr, 0op/s rd, 4op/s wr
</code></pre>

<p>ceph -s 查看集群状态为HEALTH_WARN，pgs状态active+undersized。说明有osd pool里面有一个pg副本数少于集群副本数。<br />
ceph -s health detail 查看问题详情。<br />
ceph pg dump_stuck undersized 查看undersized的pgid。<br />
ceph pg repair $pgid 尝试修复这个pgid。<br />
我这里并没有修复成功，所以删除了这个osd pool（数据会丢失！），所以一开始创建osd pool时候通过ceph -s 查看集群状态是否是HEALTH_OK，如果不是，不要存储数据，保证集群状态正常之后再使用。<br />
恢复命令如下：</p>

<pre><code>所有节点添加配置,并重启
echo '[mon]' &gt;&gt; /etc/ceph/ceph.conf
echo 'mon allow pool delete = true' &gt;&gt; /etc/ceph/ceph.conf
systemctl restart ceph-mon.target
删除data和metadata
ceph osd pool delete cephfs_data cephfs_data --yes-i-really-really-mean-it
ceph osd pool delete cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it
从新创建
ceph osd pool create cephfs_data 128
ceph osd pool create cephfs_metadata 64
ceph fs new mycephfs cephfs_metadata cephfs_data
查看详情
ceph -s 
ceph osd dump
ceph pg ls
注：虽然从新创建的osd pool，但是osd的编号不重复，可以通过ceph osd dump查看，并且pgid是由osd编号生成的，也是不重复的。
</code></pre>

<h4 id="通过ceph-fuse挂载">通过ceph fuse挂载</h4>

<p>挂载上之后可以往挂载点写入数据并通过ceph df 查看数据变化</p>

<pre><code>yum install -y ceph-fuse.x86_64
mkdir /mnt/cephfs
ceph-fuse -k /etc/ceph/ceph.client.admin.keyring -m ceph-node1:6789 /mnt/cephfs
ceph df
</code></pre>

<h4 id="通过ceph-kernel-driver挂载">通过ceph kernel driver挂载</h4>

<p>其中name和secret在/etc/ceph/ceph.client.admin.keyring文件中</p>

<pre><code>mkdir /mnt/kcephfs
mount -t ceph ceph-node1:6789:/ /mnt/kcephfs -o name=admin,secret=AQADZ2Zct/XsIBAAEIKMKWaZva/crzlrCGSUkQ==
</code></pre>

<h3 id="增加rbd">增加rbd</h3>

<p>在集群内的任意server上创建一个pool，并init：</p>

<pre><code>ceph osd pool create rbd_pool 20 20
rbd pool init rbd_pool
</code></pre>

<p>创建块设备并挂载</p>

<pre><code>rbd create testrbd --size 10240 --pool rbd_pool
rbd map testrbd -p rbd_pool
rbd info rbd_pool/testrbd
rbd feature disable rbd_pool/testrbd fast-diff
rbd feature disable rbd_pool/testrbd deep-flatten
rbd feature disable rbd_pool/testrbd object-map
rbd feature disable rbd_pool/testrbd exclusive-lock
rbd info rbd_pool/testrbd
rbd map testrbd -p rbd_pool
mkfs.ext4  /dev/rbd0
mkdir /mnt/rbd/
mount  /dev/rbd0 /mnt/rbd/
</code></pre>


                
                
<div class="entry-shang text-center">
    
	    <p>「真诚赞赏，手留余香」</p>
	
	<button class="zs show-zs btn btn-bred">赞赏支持</button>
</div>
<div class="zs-modal-bg"></div>
<div class="zs-modal-box">
	<div class="zs-modal-head">
		<button type="button" class="close">×</button>
		<span class="author"><a href="https://masxhub.gitee.io/blog"><img src="/blog/img/favicon.png" />Mashaoxun Blog</a></span>
        
	        <p class="tip"><i></i><span>真诚赞赏，手留余香</span></p>
		
 
	</div>
	<div class="zs-modal-body">
		<div class="zs-modal-btns">
			<button class="btn btn-blink" data-num="2">2元</button>
			<button class="btn btn-blink" data-num="5">5元</button>
			<button class="btn btn-blink" data-num="10">10元</button>
			<button class="btn btn-blink" data-num="20">20元</button>
			<button class="btn btn-blink" data-num="50">50元</button>
			<button class="btn btn-blink" data-num="100">100元</button>
		</div>
		<div class="zs-modal-pay">
			<button class="btn btn-bred" id="pay-text">2元</button>
			<p>使用<span id="pay-type">微信</span>扫描二维码完成支付</p>
			<img src="/blog/img/reward/zanshang.jpg"  id="pay-image"/>
		</div>
	</div>
	<div class="zs-modal-footer">
		<label><input type="radio" name="zs-type" value="wechat" class="zs-type" checked="checked"><span ><span class="zs-wechat"><img src="/blog/img/reward/wechat-btn.png"/></span></label>
		<label><input type="radio" name="zs-type" value="alipay" class="zs-type" class="zs-alipay"><img src="/blog/img/reward/alipay-btn.png"/></span></label>
	</div>
</div>
<script type="text/javascript" src="/blog/js/reward.js"></script>

                

                <hr>
                <ul class="pager">
                    
                    <li class="previous">
                        <a href="2019/01/29/ceph-profile/" data-toggle="tooltip" data-placement="top" title="ceph简介">&larr;
                            Previous Post</a>
                    </li>
                    
                    
                    <li class="next">
                        <a href="/blog/post/2019-02-21-checkdomainssl/" data-toggle="tooltip" data-placement="top" title="SSL域名证书过期监控">Next
                            Post &rarr;</a>
                    </li>
                    
                </ul>

                
<div id="disqus-comment"></div>



            </div>
            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                sidebar-container">

                
                
                <section>
                    <hr class="hidden-sm hidden-xs">
                    <h5><a href="/tags/">FEATURED TAGS</a></h5>
                    <div class="tags">
                        
                        
                        
                        <a href="/blog/tags/ceph" title="ceph">
                            ceph
                        </a>
                        
                        
                        
                        
                        
                        <a href="/blog/tags/docker" title="docker">
                            docker
                        </a>
                        
                        
                        
                        <a href="/blog/tags/k8s" title="k8s">
                            k8s
                        </a>
                        
                        
                        
                        
                        
                        
                        
                        
                        
                        <a href="/blog/tags/%E5%AD%98%E5%82%A8" title="存储">
                            存储
                        </a>
                        
                        
                        
                        <a href="/blog/tags/%E7%9B%91%E6%8E%A7" title="监控">
                            监控
                        </a>
                        
                        
                        
                        
                    </div>
                </section>
                

                
                
                <section>
                    <hr>
                    <h5>FRIENDS</h5>
                    <ul class="list-inline">
                        
                        <li><a target="_blank" href="http://www.zrq.org.cn">瑞卿博客</a></li>
                        
                    </ul>
                </section>
                
            </div>
        </div>
    </div>
</article>




<footer>
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <ul class="list-inline text-center">
                   
                    
                    <li>
                        <a href="mailto:youremail@gmail.com">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-envelope fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    
                    
                    
                    
                    

                    

		    
                    
                    <li>
                        <a target="_blank" href="/blog/img/404.jpeg">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-wechat fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    
                    
                    <li>
                        <a target="_blank" href="https://github.com/yourgithub">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-github fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    
                    
                    
                </ul>
		<p class="copyright text-muted">
                    Copyright &copy; Mashaoxun Blog , 2019
                    <br>
                    <a href="https://themes.gohugo.io/hugo-theme-cleanwhite">CleanWhite Hugo Theme</a> by <a href="https://zhaohuabing.com">Huabing</a> |
                    <iframe
                        style="margin-left: 2px; margin-bottom:-5px;"
                        frameborder="0" scrolling="0" width="100px" height="20px"
                        src="https://ghbtns.com/github-btn.html?user=zhaohuabing&repo=hugo-theme-cleanwhite&type=star&count=true" >
                    </iframe>
                </p>
            </div>
        </div>
    </div>
</footer>




<script>
    function async(u, c) {
      var d = document, t = 'script',
          o = d.createElement(t),
          s = d.getElementsByTagName(t)[0];
      o.src = u;
      if (c) { o.addEventListener('load', function (e) { c(null, e); }, false); }
      s.parentNode.insertBefore(o, s);
    }
</script>






<script>
    
    if($('#tag_cloud').length !== 0){
        async("/js/jquery.tagcloud.js",function(){
            $.fn.tagcloud.defaults = {
                
                color: {start: '#bbbbee', end: '#0085a1'},
            };
            $('#tag_cloud a').tagcloud();
        })
    }
</script>


<script>
    async("https://cdnjs.cloudflare.com/ajax/libs/fastclick/1.0.6/fastclick.js", function(){
        var $nav = document.querySelector("nav");
        if($nav) FastClick.attach($nav);
    })
</script>


<script>
    (function(){
        var bp = document.createElement('script');
        var curProtocol = window.location.protocol.split(':')[0];
        if (curProtocol === 'https'){
       bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
      }
      else{
      bp.src = 'http://push.zhanzhang.baidu.com/push.js';
      }
        var s = document.getElementsByTagName("script")[0];
        s.parentNode.insertBefore(bp, s);
    })();
</script>







</body>
</html>
